package rdd.operate;


import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;

import java.util.Arrays;
import java.util.List;

//filter会产生数据倾斜，所以需要慎重考虑
public class Spark41_Operate_Filter {
    public static void main(String[] args) {
        final List<Integer> nums = Arrays.asList(1,2,3,4);
        final SparkConf conf = new SparkConf();
        conf.setMaster("local[2]");
        conf.setAppName("spark");
        final JavaSparkContext jsc = new JavaSparkContext(conf);
        final JavaRDD<Integer> rdd = jsc.parallelize(nums, 2);
//        final JavaRDD<Integer> rdd1 = rdd.filter(new Function<Integer, Boolean>() {
//            @Override
//            public Boolean call(Integer in) throws Exception {
//                return in % 2 ==0 ;
//            }
//        });

        //上一个代码可以简化为
        final JavaRDD<Integer> rdd1 = rdd.filter( num -> num % 2 ==0 );

        rdd1.collect().forEach(System.out::println);
    }
}
